{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c92e10e0",
   "metadata": {},
   "source": [
    "动作的熵加入loss,熵越大越好,这样能让模型更健壮"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUg0lEQVR4nO3dbUxb970H8K8fsAmBYwIUewysoNuqKcpDN5LASXW1qvXCOlQtCy/W3ailVZSqmYmaMkUbUpKu2Tqq7EW3bil5s5G+WJaJXaVTUdqOQUPubp3QkCERaLjdlFxQEttNGDaQ4Kfzvy9yOa0JeTAP/tv4+5GOFP/Pz/bvHONvfM7xOTYIIQSIiCQwym6AiDIXA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKSRFkCHDh3CypUrkZ2djaqqKvT09MhqhYgkkRJAf/zjH9HY2IhXX30V586dw7p161BTUwO/3y+jHSKSxCDjZNSqqips2LABv/nNbwAAmqahrKwMu3btwo9//ONkt0NEkpiT/YThcBi9vb1oamrSx4xGI1wuFzwez6z3CYVCCIVC+m1N0zA6OorCwkIYDIZF75mIEiOEwPj4OEpKSmA03nlDK+kBdO3aNcRiMdjt9rhxu92OCxcuzHqf5uZmvPbaa8loj4gW0MjICEpLS+84P+kBNBdNTU1obGzUbwcCATidToyMjEBRFImdEdFsgsEgysrKkJeXd9e6pAdQUVERTCYTfD5f3LjP54PD4Zj1PlarFVar9bZxRVEYQEQp7F67SJJ+FMxisaCyshKdnZ36mKZp6OzshKqqyW6HiCSSsgnW2NiI+vp6rF+/Hhs3bsQvf/lLTE5O4oUXXpDRDhFJIiWAvve97+Hzzz/H/v374fV68eijj+KDDz64bcc0ES1tUr4HNF/BYBA2mw2BQID7gIhS0P2+R3kuGBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRJOIBOnTqFp59+GiUlJTAYDHj33Xfj5gshsH//fnzlK1/BsmXL4HK58Nlnn8XVjI6OYtu2bVAUBfn5+di+fTsmJibmtSBElH4SDqDJyUmsW7cOhw4dmnX+wYMH8dZbb+Hw4cM4c+YMli9fjpqaGkxNTek127Ztw8DAADo6OtDe3o5Tp07hxRdfnPtSEFF6EvMAQBw/fly/rWmacDgc4he/+IU+NjY2JqxWq/jDH/4ghBBicHBQABCffPKJXvP+++8Lg8EgLl++fF/PGwgEBAARCATm0z4RLZL7fY8u6D6gixcvwuv1wuVy6WM2mw1VVVXweDwAAI/Hg/z8fKxfv16vcblcMBqNOHPmzKyPGwqFEAwG4yYiSn8LGkBerxcAYLfb48btdrs+z+v1ori4OG6+2WxGQUGBXjNTc3MzbDabPpWVlS1k20QkSVocBWtqakIgENCnkZER2S0R0QJY0AByOBwAAJ/PFzfu8/n0eQ6HA36/P25+NBrF6OioXjOT1WqFoihxExGlvwUNoPLycjgcDnR2dupjwWAQZ86cgaqqAABVVTE2Nobe3l69pqurC5qmoaqqaiHbIaIUZ070DhMTE/jHP/6h37548SL6+vpQUFAAp9OJ3bt342c/+xkeeughlJeXY9++fSgpKcGWLVsAAI888gi+9a1vYceOHTh8+DAikQgaGhrwzDPPoKSkZMEWjIjSQKKH1z766CMB4Lapvr5eCHHrUPy+ffuE3W4XVqtVPPnkk2JoaCjuMa5fvy6+//3vi9zcXKEoinjhhRfE+Pj4gh/iIyI57vc9ahBCCIn5NyfBYBA2mw2BQID7g4hS0P2+R9PiKBgRLU0MICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEiahH+Wh2gxxMI3MTZ8HhDarQGDAbay1TBbc+Q2RouKAUQpITR+HRdPtkLEogAAg9GEiq17GUBLHDfBKCXEIqFbvzBHGYUBRClBi4TABMo8DCBKCbHwDXz5NzINBiMMRoPEjigZGECUEm7+68oXO6ABZC1fAXN2nsSOKBkYQJQSZv5CuNGcBaOJx0iWOgYQpSSD0QyD0SS7DVpkCQVQc3MzNmzYgLy8PBQXF2PLli0YGhqKq5mamoLb7UZhYSFyc3NRV1cHn88XVzM8PIza2lrk5OSguLgYe/bsQTQanf/SUFqa+ekHAIwmBlAmSCiAuru74Xa7cfr0aXR0dCASiWDz5s2YnJzUa1555RW89957aGtrQ3d3N65cuYKtW7fq82OxGGpraxEOh/Hxxx/jnXfewZEjR7B///6FWypKO9Pf/9EZjLcmWtrEPPj9fgFAdHd3CyGEGBsbE1lZWaKtrU2v+fTTTwUA4fF4hBBCnDhxQhiNRuH1evWalpYWoSiKCIVC9/W8gUBAABCBQGA+7VOK0LSYuPRfvxc9h3fo0+C7B4WmabJbozm63/fovP6LCQQCAICCggIAQG9vLyKRCFwul16zatUqOJ1OeDweAIDH48GaNWtgt9v1mpqaGgSDQQwMDMz6PKFQCMFgMG6iJUQAsUhYdhckwZwDSNM07N69G4899hhWr14NAPB6vbBYLMjPz4+rtdvt8Hq9es2Xw2d6/vS82TQ3N8Nms+lTWVnZXNumlCSgRabiRgxGbn5lgjm/ym63G+fPn8exY8cWsp9ZNTU1IRAI6NPIyMiiPyclj9Bit74H9CU5haWSuqFkmtMXLRoaGtDe3o5Tp06htPSLPxSHw4FwOIyxsbG4T0E+nw8Oh0Ov6enpiXu86aNk0zUzWa1WWK3WubRKaUAIAaHF4sZM1uWSuqFkSugTkBACDQ0NOH78OLq6ulBeXh43v7KyEllZWejs7NTHhoaGMDw8DFVVAQCqqqK/vx9+v1+v6ejogKIoqKiomM+y0BJiyuJ/OJkgoU9AbrcbR48exZ///Gfk5eXp+2xsNhuWLVsGm82G7du3o7GxEQUFBVAUBbt27YKqqqiurgYAbN68GRUVFXj22Wdx8OBBeL1e7N27F263m59ySGc0828hEyQUQC0tLQCAxx9/PG68tbUVzz//PADgzTffhNFoRF1dHUKhEGpqavD222/rtSaTCe3t7di5cydUVcXy5ctRX1+PAwcOzG9JKG0Jod1+KoaJX0LMBAYx85VPA8FgEDabDYFAAIqiyG6H5ilyM4jB/3wd4cl/6WPljz+Pooc3SeyK5uN+36M81knSabEIhIjdu5CWHAYQSSeiUQhNu3chLTkMIJIuPDGKaOiGfttotsCqPCCxI0oWBhBJp2nRuIuRGYwmmHgx+ozAAKLUYzDCaLbI7oKSgAFEKcdgMMBoypLdBiUBA4ikm3kaBgwGXowsQzCASLpYJCS7BZKEAUTSzbwUB2UOBhBJF52ajLttMJpgMPA3wTIBA4iku3E9/vpO2TY7j4JlCAYQyXfbb4JZeUH6DMFXmVKO0WzhJliGYACRVLNdjMFozuInoAzBV5mkEyL+RFT+JHPmYACRVEKLQYtGbhvnJlhmYACRXEKDFrs9gCgzMIBIKqFp0KIzvgnN/T8Zg680SRUL30QoeC1ubFnBVyV1Q8nGACKphNBmnIxqgJnXAsoYDCBKOfxJnszBAKKUwx8lzBwMIJJrxneAYPj/LyJSRmAAkVSxaPi2c8Eoc/Arp7SootEoJiYm7jh/amw0/pvQApgYn0DUOjZrvcViQU4Od1IvFQwgWlR9fX3YunUrtDv87tfqlYXY98x6ZJlvXYJVCA3/sW0b/ufy2Kz1zz33HH7+858vVruUZAwgWlThcBiXL1++YwBV/ZsNvujDuHLjYeSZRlEQ/W8M/XMYV66Pz1o/Nja2iN1SsiW0D6ilpQVr166FoihQFAWqquL999/X509NTcHtdqOwsBC5ubmoq6uDz+eLe4zh4WHU1tYiJycHxcXF2LNnD6LR6MIsDaWdAB7E4OS/YzTyVfzv1GqcG92IGyH+PWSKhAKotLQUb7zxBnp7e3H27Fk88cQT+M53voOBgQEAwCuvvIL33nsPbW1t6O7uxpUrV7B161b9/rFYDLW1tQiHw/j444/xzjvv4MiRI9i/f//CLhWljZuxPGiYPuplwL/CCsJR/kxzpkhoE+zpp5+Ou/3666+jpaUFp0+fRmlpKX7729/i6NGjeOKJJwAAra2teOSRR3D69GlUV1fjL3/5CwYHB/HXv/4Vdrsdjz76KH7605/iRz/6EX7yk5/AYuFlODNNYdZl5BgDuKEpMCKKYtMFRGP8BJQp5rwPKBaLoa2tDZOTk1BVFb29vYhEInC5XHrNqlWr4HQ64fF4UF1dDY/HgzVr1sBut+s1NTU12LlzJwYGBvC1r30toR4uXLiA3NzcuS4CJcGlS5dmvejYtH9eGoC16wA+D5dhuSkAw9RniERjd6wfHR3F4ODgYrRKC+huRz6/LOEA6u/vh6qqmJqaQm5uLo4fP46Kigr09fXBYrEgPz8/rt5ut8Pr9QIAvF5vXPhMz5+edyehUAih0BdnTAeDQQBAIBDg/qMUd68/xMFLn2Pw0uf3/XjhcJg7otPA5OTkvYswhwB6+OGH0dfXh0AggD/96U+or69Hd3d3wg0morm5Ga+99tpt41VVVVAUZVGfm+bPYDDc9VNQIhwOBzZt2rQgj0WLZ/pDwr0k/E1oi8WCBx98EJWVlWhubsa6devwq1/9Cg6HY9b/nXw+HxwOB4Bbfzwzj4pN356umU1TUxMCgYA+jYyM3LGWiNLHvE/F0DQNoVAIlZWVyMrKQmdnpz5vaGgIw8PDUFUVAKCqKvr7++H3+/Wajo4OKIqCioqKOz6H1WrVD/1PT0SU/hLaBGtqasJTTz0Fp9OJ8fFxHD16FCdPnsSHH34Im82G7du3o7GxEQUFBVAUBbt27YKqqqiurgYAbN68GRUVFXj22Wdx8OBBeL1e7N27F263G1Yrz4AmyjQJBZDf78dzzz2Hq1evwmazYe3atfjwww/xzW9+EwDw5ptvwmg0oq6uDqFQCDU1NXj77bf1+5tMJrS3t2Pnzp1QVRXLly9HfX09Dhw4sLBLRSnDZDJBUZQ7fhM6UdnZ2QvyOJQaDGKh9g4mUTAYhM1mQyAQ4OZYiguFQnGb3POVm5uLFStWLNjj0eK43/cozwWjRWW1WlFWVia7DUpRvB4QEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAiksYsu4G5EEIAAILBoOROiGg20+/N6ffqnaRlAF2/fh0AUFZWJrkTIrqb8fFx2Gy2O85PywAqKCgAAAwPD9914SheMBhEWVkZRkZGoCiK7HbSAtfZ3AghMD4+jpKSkrvWpWUAGY23dl3ZbDb+UcyBoihcbwniOkvc/Xw44E5oIpKGAURE0qRlAFmtVrz66quwWq2yW0krXG+J4zpbXAZxr+NkRESLJC0/ARHR0sAAIiJpGEBEJA0DiIikScsAOnToEFauXIns7GxUVVWhp6dHdkvSNDc3Y8OGDcjLy0NxcTG2bNmCoaGhuJqpqSm43W4UFhYiNzcXdXV18Pl8cTXDw8Oora1FTk4OiouLsWfPHkSj0WQuijRvvPEGDAYDdu/erY9xnSWJSDPHjh0TFotF/O53vxMDAwNix44dIj8/X/h8PtmtSVFTUyNaW1vF+fPnRV9fn/j2t78tnE6nmJiY0GteeuklUVZWJjo7O8XZs2dFdXW12LRpkz4/Go2K1atXC5fLJf7+97+LEydOiKKiItHU1CRjkZKqp6dHrFy5Uqxdu1a8/PLL+jjXWXKkXQBt3LhRuN1u/XYsFhMlJSWiublZYlepw+/3CwCiu7tbCCHE2NiYyMrKEm1tbXrNp59+KgAIj8cjhBDixIkTwmg0Cq/Xq9e0tLQIRVFEKBRK7gIk0fj4uHjooYdER0eH+MY3vqEHENdZ8qTVJlg4HEZvby9cLpc+ZjQa4XK54PF4JHaWOgKBAIAvTtjt7e1FJBKJW2erVq2C0+nU15nH48GaNWtgt9v1mpqaGgSDQQwMDCSx++Ryu92ora2NWzcA11kypdXJqNeuXUMsFot70QHAbrfjwoULkrpKHZqmYffu3XjsscewevVqAIDX64XFYkF+fn5crd1uh9fr1WtmW6fT85aiY8eO4dy5c/jkk09um8d1ljxpFUB0d263G+fPn8ff/vY32a2ktJGREbz88svo6OhAdna27HYyWlptghUVFcFkMt12NMLn88HhcEjqKjU0NDSgvb0dH330EUpLS/Vxh8OBcDiMsbGxuPovrzOHwzHrOp2et9T09vbC7/fj61//OsxmM8xmM7q7u/HWW2/BbDbDbrdznSVJWgWQxWJBZWUlOjs79TFN09DZ2QlVVSV2Jo8QAg0NDTh+/Di6urpQXl4eN7+yshJZWVlx62xoaAjDw8P6OlNVFf39/fD7/XpNR0cHFEVBRUVFchYkiZ588kn09/ejr69Pn9avX49t27bp/+Y6SxLZe8ETdezYMWG1WsWRI0fE4OCgePHFF0V+fn7c0YhMsnPnTmGz2cTJkyfF1atX9enGjRt6zUsvvSScTqfo6uoSZ8+eFaqqClVV9fnTh5Q3b94s+vr6xAcffCAeeOCBjDqk/OWjYEJwnSVL2gWQEEL8+te/Fk6nU1gsFrFx40Zx+vRp2S1JA2DWqbW1Va+5efOm+MEPfiBWrFghcnJyxHe/+11x9erVuMe5dOmSeOqpp8SyZctEUVGR+OEPfygikUiSl0aemQHEdZYcvBwHEUmTVvuAiGhpYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETS/B/iC9mR39brYQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (3): ReLU()\n",
       "  (4): Linear(in_features=64, out_features=2, bias=True)\n",
       "  (5): Softmax(dim=1)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,计算每个动作的概率\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    "    torch.nn.Softmax(dim=1),\n",
    ")\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_958/449353166.py:30: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor(state).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-987.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    state = []\n",
    "    action = []\n",
    "    reward = []\n",
    "\n",
    "    s = env.reset()\n",
    "    o = False\n",
    "    while not o:\n",
    "        #根据概率采样\n",
    "        prob = model(torch.FloatTensor(s).reshape(1, 4))[0].tolist()\n",
    "        a = random.choices(range(2), weights=prob, k=1)[0]\n",
    "\n",
    "        ns, r, o = env.step(a)\n",
    "\n",
    "        state.append(s)\n",
    "        action.append(a)\n",
    "        reward.append(r)\n",
    "\n",
    "        s = ns\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    state = torch.FloatTensor(state).reshape(-1, 4)\n",
    "    action = torch.LongTensor(action).reshape(-1, 1)\n",
    "    reward = torch.FloatTensor(reward).reshape(-1, 1)\n",
    "\n",
    "    return state, action, reward, reward.sum().item()\n",
    "\n",
    "\n",
    "state, action, reward, reward_sum = play()\n",
    "\n",
    "reward_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dbe2cd7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 -590.851318359375 0.33475616574287415 -983.05\n",
      "100 14.871795654296875 0.2545207142829895 200.0\n",
      "200 15.10959243774414 0.2453250139951706 96.6\n",
      "300 15.055750846862793 0.23422563076019287 200.0\n",
      "400 14.361923217773438 0.2557045817375183 200.0\n",
      "500 15.0393648147583 0.245320662856102 145.25\n",
      "600 15.268821716308594 0.2585984170436859 200.0\n",
      "700 15.385765075683594 0.2580231726169586 92.6\n",
      "800 15.595657348632812 0.25203099846839905 144.8\n",
      "900 14.314332962036133 0.24231958389282227 200.0\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=5e-3)\n",
    "\n",
    "    #训练N局\n",
    "    for epoch in range(1000):\n",
    "\n",
    "        #一个epoch最少玩N步\n",
    "        steps = 0\n",
    "        while steps < 200:\n",
    "\n",
    "            #玩一局游戏,得到数据\n",
    "            state, action, reward, _ = play()\n",
    "            steps += len(state)\n",
    "\n",
    "            #计算当前state的价值,其实就是Q(state,action),这里是用蒙特卡洛法估计的\n",
    "            value = []\n",
    "            for i in range(len(reward)):\n",
    "                s = 0\n",
    "                for j in range(i, len(reward)):\n",
    "                    s += reward[j] * 0.99**(j - i)\n",
    "                value.append(s)\n",
    "            value = torch.FloatTensor(value).reshape(-1, 1)\n",
    "\n",
    "            #重新计算动作的概率\n",
    "            prob = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #求Q最大的导函数 -> partial value / partial action\n",
    "            loss = (prob + 1e-8).log() * value\n",
    "            for i in range(len(loss)):\n",
    "                loss[i] = loss[i] * 0.99**i\n",
    "            loss = -loss.mean()\n",
    "\n",
    "            #计算动作的熵,越大越好\n",
    "            entropy = prob * (prob + 1e-8).log()\n",
    "            entropy = -entropy.mean()\n",
    "            loss -= entropy * 5\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, loss.item(), entropy.item(), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATm0lEQVR4nO3dbUxb970H8K8N2ISQYwIMuwysRFu0hJuHbiSB015pU+uGdahaVl50U9Sy3ihVMxM1ZYo2pDRZuwd6sxfduiXkzZb0TUovlbLdovSBkpbc3prQuEUiJEGtlAjaxnZbxjHQYh78uy8qzq0b0voJ/3Hy/UhHiv//n+3fObG/nAdsLCIiICJSwKq6ASK6eTGAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGWUBdPjwYaxatQr5+fmoqalBX1+fqlaISBElAfTcc8+hubkZBw8exNtvv41Nmzahrq4OoVBIRTtEpIhFxYdRa2pqsGXLFvz1r38FAESjUVRWVmLPnj349a9/nel2iEiR3Ew/4fT0NPx+P1paWswxq9UKj8cDn8+34H0ikQgikYh5OxqNYnR0FCUlJbBYLIveMxElRkQwPj6O8vJyWK3XP9DKeAB9/PHHmJubg9PpjBl3Op24dOnSgvdpbW3F448/non2iCiNRkZGUFFRcd35jAdQMlpaWtDc3GzeNgwDbrcbIyMj0DRNYWdEtJBwOIzKykqsWLHiK+syHkClpaXIyclBMBiMGQ8Gg3C5XAvex263w263XzOuaRoDiGgJ+7pTJBm/Cmaz2VBdXY3u7m5zLBqNoru7G7quZ7odIlJIySFYc3MzGhsbsXnzZmzduhV/+tOfMDk5iQcffFBFO0SkiJIAuu+++/DRRx/hwIEDCAQCuPXWW/HSSy9dc2KaiG5sSn4PKFXhcBgOhwOGYfAcENESFO97lJ8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIJB9CZM2dwzz33oLy8HBaLBf/4xz9i5kUEBw4cwC233IJly5bB4/Hg3XffjakZHR3Fjh07oGkaioqKsHPnTkxMTKS0IkSUfRIOoMnJSWzatAmHDx9ecP7QoUN4+umncfToUZw9exbLly9HXV0dpqamzJodO3ZgcHAQXV1d6OzsxJkzZ/DQQw8lvxZElJ0kBQDk5MmT5u1oNCoul0v++Mc/mmNjY2Nit9vl2WefFRGRCxcuCAB56623zJoXX3xRLBaLfPDBB3E9r2EYAkAMw0ilfSJaJPG+R9N6Dujy5csIBALweDzmmMPhQE1NDXw+HwDA5/OhqKgImzdvNms8Hg+sVivOnj274ONGIhGEw+GYhYiyX1oDKBAIAACcTmfMuNPpNOcCgQDKyspi5nNzc1FcXGzWfFlrayscDoe5VFZWprNtIlIkK66CtbS0wDAMcxkZGVHdEhGlQVoDyOVyAQCCwWDMeDAYNOdcLhdCoVDM/OzsLEZHR82aL7Pb7dA0LWYhouyX1gBavXo1XC4Xuru7zbFwOIyzZ89C13UAgK7rGBsbg9/vN2tOnz6NaDSKmpqadLZDREtcbqJ3mJiYwHvvvWfevnz5Mvr7+1FcXAy32429e/fid7/7HdasWYPVq1fjscceQ3l5ObZv3w4AWLduHX74wx9i165dOHr0KGZmZtDU1ISf/vSnKC8vT9uKEVEWSPTy2muvvSYArlkaGxtF5PNL8Y899pg4nU6x2+1y5513ytDQUMxjfPLJJ/Kzn/1MCgsLRdM0efDBB2V8fDztl/iISI1436MWERGF+ZeUcDgMh8MBwzB4PohoCYr3PZoVV8GI6MbEACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUSfjP8tDNSUQQMUKY/OiKOZZjK4DD/W+wWPhzjJLDAKK4Ge8PYvh/283by1aWQ6tYB0sOA4iSw1cOxS0L/4ITLXEMIIofA4jSjAFE8WMAUZoxgChugqjqFugGwwCi+HEPiNKMAURx40loSreEAqi1tRVbtmzBihUrUFZWhu3bt2NoaCimZmpqCl6vFyUlJSgsLERDQwOCwWBMzfDwMOrr61FQUICysjLs27cPs7Ozqa8NLS4GEKVZQgHU09MDr9eL3t5edHV1YWZmBtu2bcPk5KRZ8+ijj+KFF15AR0cHenp68OGHH+Lee+815+fm5lBfX4/p6Wm8+eabeOaZZ3D8+HEcOHAgfWtFi4MBROkmKQiFQgJAenp6RERkbGxM8vLypKOjw6y5ePGiABCfzyciIqdOnRKr1SqBQMCsaWtrE03TJBKJxPW8hmEIADEMI5X2KQHRaFTeP/ff0nd0l7kMPHdQ5mZnVLdGS1C879GUzgEZhgEAKC4uBgD4/X7MzMzA4/GYNWvXroXb7YbP5wMA+Hw+bNiwAU6n06ypq6tDOBzG4ODggs8TiUQQDodjFlKAe0CUZkkHUDQaxd69e3H77bdj/fr1AIBAIACbzYaioqKYWqfTiUAgYNZ8MXzm5+fnFtLa2gqHw2EulZWVybZNKRAGEKVZ0gHk9Xpx/vx5tLe3f31xilpaWmAYhrmMjIws+nPSAhhAlGZJfRi1qakJnZ2dOHPmDCoqKsxxl8uF6elpjI2NxewFBYNBuFwus6avry/m8eavks3XfJndbofdbk+mVUonBhClWUJ7QCKCpqYmnDx5EqdPn8bq1atj5qurq5GXl4fu7m5zbGhoCMPDw9B1HQCg6zoGBgYQCoXMmq6uLmiahqqqqlTWhRaZgAFE6ZXQHpDX68WJEyfwz3/+EytWrDDP2TgcDixbtgwOhwM7d+5Ec3MziouLoWka9uzZA13XUVtbCwDYtm0bqqqqcP/99+PQoUMIBALYv38/vF4v93KWOuFHMSi9EgqgtrY2AMAPfvCDmPFjx47h5z//OQDgqaeegtVqRUNDAyKRCOrq6nDkyBGzNicnB52dndi9ezd0Xcfy5cvR2NiIJ554IrU1oUXHk9CUbhbJwldVOByGw+GAYRjQNE11OzcFEcGIrwPBgVfNsWUry1HVsB/WHH6vHcWK9z3Kz4JR3LLwZxUtcQwgSgADiNKLAUTx40loSjMGEMWNh2CUbgwgih8DiNKMAURxEkh0LnaIfw+MUsRXEMVHgM/GrsYM5Re5+EcJKSV89VDcvrwHxN//oVQxgCh5FovqDijLMYAoaRYGEKWIAUQpYABRahhAlDyLlRlEKWEAUdJ4CEapYgBR8hhAlCIGECWNe0CUKgYQpYABRKlhAFHyuAdEKWIAUdI+/xgGQ4iSxwCi5HEPiFLEAKKk8SQ0pYoBRClgAFFqGECUPO4BUYoYQJQ0HoJRqhhAlDx+GRmliN8oRaZPP/0U09PTC86JRDE3G/uFZJFIBIYxhuudCyosLERuLl9idH18dZDpD3/4A44fP77gnNViwX/+h45v3VJkjh0+fAT/9T/NC9ZbLBY8//zzqKmpWYRO6UbBACKTYRj44IMPFpyzWiyYjFhxabIW4bkSfNP+LsbC/uvWWyyW6+5NEc1L6CC+ra0NGzduhKZp0DQNuq7jxRdfNOenpqbg9XpRUlKCwsJCNDQ0IBgMxjzG8PAw6uvrUVBQgLKyMuzbtw+zs7PpWRtaRBZcnNRxZWoDRme+icGJf0dgapXqpijLJRRAFRUVePLJJ+H3+3Hu3Dnccccd+PGPf4zBwUEAwKOPPooXXngBHR0d6OnpwYcffoh7773XvP/c3Bzq6+sxPT2NN998E8888wyOHz+OAwcOpHetKO0EwMTsSsyf74kiFxOzhUp7ouyX0CHYPffcE3P797//Pdra2tDb24uKigr87W9/w4kTJ3DHHXcAAI4dO4Z169aht7cXtbW1eOWVV3DhwgW8+uqrcDqduPXWW/Hb3/4Wv/rVr/Cb3/wGNpstfWtGaWWBoDz/XeRgDeaQiwJrGCW576tui7Jc0ueA5ubm0NHRgcnJSei6Dr/fj5mZGXg8HrNm7dq1cLvd8Pl8qK2thc/nw4YNG+B0Os2auro67N69G4ODg/jud7+bUA+XLl1CYSF/CqfL6OjodedEBH29HcCydzA5V4RS2/t47/LFr3y8K1euoKSkJN1tUhaYmJiIqy7hABoYGICu65iamkJhYSFOnjyJqqoq9Pf3w2azoaioKKbe6XQiEAgAAAKBQEz4zM/Pz11PJBJBJBIxb4fDYQCfnzTl+aP0+eI2/jIBcKp3CMBQ3I83MTGBsbGxlPui7DM5ORlXXcIB9J3vfAf9/f0wDAPPP/88Ghsb0dPTk3CDiWhtbcXjjz9+zXhNTQ00TVvU576ZPPvss2l9vPXr1+O2225L62NSdpjfSfg6Cf8qq81mw7e//W1UV1ejtbUVmzZtwp///Ge4XC5MT09f8xMvGAzC5XIBAFwu1zVXxeZvz9cspKWlBYZhmMvIyEiibRPREpTy79JHo1FEIhFUV1cjLy8P3d3d5tzQ0BCGh4eh6zoAQNd1DAwMIBQKmTVdXV3QNA1VVVXXfQ673W5e+p9fiCj7JXQI1tLSgrvvvhtutxvj4+M4ceIEXn/9dbz88stwOBzYuXMnmpubUVxcDE3TsGfPHui6jtraWgDAtm3bUFVVhfvvvx+HDh1CIBDA/v374fV6YbfbF2UFiWjpSiiAQqEQHnjgAVy9ehUOhwMbN27Eyy+/jLvuugsA8NRTT8FqtaKhoQGRSAR1dXU4cuSIef+cnBx0dnZi9+7d0HUdy5cvR2NjI5544on0rhUlJT8/P217lxaLBTk5OWl5LLpxWUREVDeRqHA4DIfDAcMweDiWRv/617/ivnwaj7KyMu7Z3qTifY/ys2BkWrlyJVauXKm6DbqJ8AtdiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTK5qhtIhogAAMLhsOJOiGgh8+/N+ffq9WRlAH3yyScAgMrKSsWdENFXGR8fh8PhuO58VgZQcXExAGB4ePgrV45ihcNhVFZWYmRkBJqmqW4nK3CbJUdEMD4+jvLy8q+sy8oAslo/P3XlcDj4okiCpmncbgniNktcPDsHPAlNRMowgIhImawMILvdjoMHD8Jut6tuJatwuyWO22xxWeTrrpMRES2SrNwDIqIbAwOIiJRhABGRMgwgIlImKwPo8OHDWLVqFfLz81FTU4O+vj7VLSnT2tqKLVu2YMWKFSgrK8P27dsxNDQUUzM1NQWv14uSkhIUFhaioaEBwWAwpmZ4eBj19fUoKChAWVkZ9u3bh9nZ2UyuijJPPvkkLBYL9u7da45xm2WIZJn29nax2Wzy97//XQYHB2XXrl1SVFQkwWBQdWtK1NXVybFjx+T8+fPS398vP/rRj8TtdsvExIRZ8/DDD0tlZaV0d3fLuXPnpLa2Vm677TZzfnZ2VtavXy8ej0feeecdOXXqlJSWlkpLS4uKVcqovr4+WbVqlWzcuFEeeeQRc5zbLDOyLoC2bt0qXq/XvD03Nyfl5eXS2tqqsKulIxQKCQDp6ekREZGxsTHJy8uTjo4Os+bixYsCQHw+n4iInDp1SqxWqwQCAbOmra1NNE2TSCSS2RXIoPHxcVmzZo10dXXJ97//fTOAuM0yJ6sOwaanp+H3++HxeMwxq9UKj8cDn8+nsLOlwzAMAP//gV2/34+ZmZmYbbZ27Vq43W5zm/l8PmzYsAFOp9OsqaurQzgcxuDgYAa7zyyv14v6+vqYbQNwm2VSVn0Y9eOPP8bc3FzMfzoAOJ1OXLp0SVFXS0c0GsXevXtx++23Y/369QCAQCAAm82GoqKimFqn04lAIGDWLLRN5+duRO3t7Xj77bfx1ltvXTPHbZY5WRVA9NW8Xi/Onz+PN954Q3UrS9rIyAgeeeQRdHV1IT8/X3U7N7WsOgQrLS1FTk7ONVcjgsEgXC6Xoq6WhqamJnR2duK1115DRUWFOe5yuTA9PY2xsbGY+i9uM5fLteA2nZ+70fj9foRCIXzve99Dbm4ucnNz0dPTg6effhq5ublwOp3cZhmSVQFks9lQXV2N7u5ucywajaK7uxu6rivsTB0RQVNTE06ePInTp09j9erVMfPV1dXIy8uL2WZDQ0MYHh42t5mu6xgYGEAoFDJrurq6oGkaqqqqMrMiGXTnnXdiYGAA/f395rJ582bs2LHD/De3WYaoPgueqPb2drHb7XL8+HG5cOGCPPTQQ1JUVBRzNeJmsnv3bnE4HPL666/L1atXzeXTTz81ax5++GFxu91y+vRpOXfunOi6Lrqum/Pzl5S3bdsm/f398tJLL8k3vvGNm+qS8hevgolwm2VK1gWQiMhf/vIXcbvdYrPZZOvWrdLb26u6JWUALLgcO3bMrPnss8/kF7/4haxcuVIKCgrkJz/5iVy9ejXmca5cuSJ33323LFu2TEpLS+WXv/ylzMzMZHht1PlyAHGbZQa/joOIlMmqc0BEdGNhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMr8H1Lza9uBUma4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
